# -*- coding:utf-8 -*-
# @Time     : 2021/4/6 15:49
import json
import time
import requests
import random
import re
from pymongo import MongoClient
from lxml import etree
from redis import Redis
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor

requests.packages.urllib3.disable_warnings()
count = 0
client = MongoClient('mongodb://root:MgXwM3Toe5@192.168.7.53:30228')
baiduwenku_url_ = client["webpage"]["baiduwenku_url_"]
proxies = client["proxy"]["proxies_ip_dongtai"]
# 获取类型 txt/doc/pdf...
comp = re.compile('"typeName":"(.*?)"')
# 获取标题
comp2 = re.compile('<title>(.*?)</title>')
# 获取多个jsonUrl
comp3 = re.compile(r'\\"(https:.*?0.json?.*?)\\"}')

db = Redis(
    host="192.168.7.52",
    port=30114,
    db=15,
    decode_responses=True,
    password="3oSama4la]uLjzO1")


# 返回随机ua
def get_ua():
    first_num = random.randint(55, 62)
    third_num = random.randint(0, 3200)
    fourth_num = random.randint(0, 140)
    os_type = [
        '(Windows NT 6.1; WOW64)', '(Windows NT 10.0; WOW64)', '(X11; Linux x86_64)',
        '(Macintosh; Intel Mac OS X 10_12_6)'
    ]
    chrome_version = 'Chrome/{}.0.{}.{}'.format(first_num, third_num, fourth_num)

    ua = ' '.join(['Mozilla/5.0', random.choice(os_type), 'AppleWebKit/537.36',
                   '(KHTML, like Gecko)', chrome_version, 'Safari/537.36']
                  )
    return ua


headers = {
    'User-Agent': get_ua(),

}


# 使用动态代理
def get_ip_dt():
    items = list(proxies.find({}))
    item = random.choice(items)
    ip = item.get("ip")
    new_proxy = {
        "http": ip,
    }
    return new_proxy


# 静态代理
proxies_list = [
    "http://pc0308wd:pc0308wd@140.246.91.53:888",
    "http://pc0308wd:pc0308wd@113.125.125.2:888",
    "http://pc0308wd:pc0308wd@203.57.232.249:888",
    "http://pc0308wd:pc0308wd@113.125.9.8:888",
    "http://pc0308wd:pc0308wd@42.123.125.53:888",
]


# 使用静态代理
def get_ip_jt():
    ip = random.choice(proxies_list)
    new_proxy = {
        "http": ip,
    }
    return new_proxy


# 使用代理的信号
ip = "dt"


# 选择使用 静态代理 或者 动态代理
def get_ip():
    if ip == "jt":
        return get_ip_jt()
    elif ip == "dt":
        return get_ip_dt()


# 解析函数
def parseUrl(url):
    try:
        resp = requests.get(url, headers=headers, timeout=5, proxies=get_ip(), verify=False)
        return resp
    except Exception as e:
        print(e, "--> parseUrl")
        return None


submit_url = "http://data-platform-api.k8s.laibokeji.com/api/data/fulltext"
submit_headers = {
    'Content-Type': 'application/json;charset=UTF-8',
    'Connection': 'close'
}


# 提交公司接口, 返回响应文本
def submit(submit_data):
    try:
        submit_resp = requests.post(submit_url,
                                    headers=submit_headers,
                                    data=submit_data,
                                    timeout=15
                                    )
        return submit_resp.text
    except:
        return None


def parseTxt(resp):
    text = resp.content.decode("utf-8")
    title = ''.join(comp2.findall(text))
    html = etree.HTML(text)
    datas = html.xpath('//p[@class="p-txt"]/text()')
    datas = str(datas)
    result = re.sub("\[|\]|, |\'", '', datas)
    title = title.split(" -")[0]
    return title, result


def parseDoc(resp):
    html = resp.text
    html = html.replace(r'\\/\\/', '/')
    html = html.replace(r'\\/', '/')
    title = ''.join(comp2.findall(html))
    urls = comp3.findall(html)
    result = ""
    for url in urls:
        url = url.replace("/w", "//w")
        resp = parseUrl(url)
        # 解决中文乱码
        text = resp.content.decode("unicode-escape")
        y = None
        datas = re.findall(r'"c":"(.*?)".*?"y":(.*?),', text)
        for data in datas:
            # 当item[1]的值与前面不同时，代表要换行了
            if not y == data[1]:
                y = data[1]
                n = '\n'
            else:
                n = ''
            result += n
            result += data[0]
    title = title.split(" -")[0]
    return title, result


def parsePdf(resp):
    html = resp.text
    html = html.replace(r'\\/\\/', '/')
    html = html.replace(r'\\/', '/')
    title = ''.join(comp2.findall(html))
    urls = comp3.findall(html)
    result = ""
    for url in urls:
        url = url.replace("/w", "//w")
        resp = parseUrl(url)
        # 解决中文乱码
        text = resp.content.decode("unicode-escape")
        # $1: 内容, $2: 是否换行的信号
        datas = re.findall(r'"c":"(.*?)".*?"y":(.*?),', text)
        y = None
        for data in datas:
            if not y == data[1]:
                y = data[1]
                n = '\n'
            else:
                n = ''
            result += n
            result += data[0]
    title = title.split(" -")[0]
    return title, result


def run(url):
    resp = parseUrl(url)
    if resp:
        text = resp.text
        file_type = "".join(comp.findall(text))
        if file_type == "txt":
            title, result = parseTxt(resp)
            print(title, len(result), url, file_type)

        elif file_type == "word":
            title, result = parseDoc(resp)
            print(title, len(result), url, file_type)
        elif file_type == "pdf":
            title, result = parsePdf(resp)
            print(title, len(result), url, file_type)

        submit_data = {
            'url': url,
            'title': title,
            'dataType': 'webpage',
            'source': 'baiduwenku',
            'content': result,
            'batchNo': time.strftime('%Y%m%d', time.localtime(time.time()))
        }
        try:
            # 提交五次
            for pan_num in range(5):
                response_end = submit(json.dumps(submit_data))
                print("响应", response_end)
                # 失败
                if 'system' in response_end or '"status":404' in response_end:
                    print("提交失败, 正在重试")
                    continue
                # 成功, 有一次成功就返回
                else:
                    if "this url is already exist" in response_end:
                        print("提交成功, 但是已存在")
                        baiduwenku_url_.update_one(
                            {"url": url},
                            {"$set": {"msg": "url exist", "status": 1}},
                        )
                    else:
                        print("提交成功")
                        baiduwenku_url_.update_one(
                            {"url": url},
                            {"$set": {"status": 2}}, )
                    break
            # 提交5次失败才会执行下方代码
            baiduwenku_url_.update_one(
                {"url": url},
                {"$set": {"status": 0}},
            )
        except:
            pass


def start():
    with ThreadPoolExecutor(max_workers=5) as pool:
        while True:
            if db.llen("urls2"):
                url = db.lpop("urls2")
                # run(url)
                pool.submit(run, url)
            else:
                print("正在等待")
                time.sleep(5)


def main():
    start()


if __name__ == '__main__':
    main()
